{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess ROCO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Statistic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dir Path\n",
    "train_dir_path = Path(\"/remote-home/share/medical/public/ROCO/train/radiology\")\n",
    "valid_dir_path = Path(\"/remote-home/share/medical/public/ROCO/valid/radiology\")\n",
    "test_dir_path = Path(\"/remote-home/share/medical/public/ROCO/test/radiology\")\n",
    "\n",
    "# Csv Path\n",
    "# test_dir_path = Path(\"/remote-home/weixionglin/vlp/Match/Preprocess\")\n",
    "train_csv_path = train_dir_path / \"traindata.csv\"\n",
    "valid_csv_path = valid_dir_path / \"valdata.csv\"\n",
    "test_csv_path = test_dir_path / \"testdata.csv\"\n",
    "\n",
    "# Data\n",
    "train_data = pd.read_csv(train_csv_path)\n",
    "valid_data = pd.read_csv(valid_csv_path)\n",
    "test_data = pd.read_csv(test_csv_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set: 65450\n",
      "Valid Set: 8180\n",
      "Test Set: 8179\n"
     ]
    }
   ],
   "source": [
    "print(f\"Train Set: {len(train_data)}\")\n",
    "print(f\"Valid Set: {len(valid_data)}\")\n",
    "print(f\"Test Set: {len(test_data)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROCO Samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>image</th>\n",
       "      <th>caption</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ROCO_00001</td>\n",
       "      <td>PMC4608653_cro-0008-0385-g01.jpg</td>\n",
       "      <td>Axial MRI (coronal view).\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROCO_00006</td>\n",
       "      <td>PMC4840839_ol-11-05-3298-g02.jpg</td>\n",
       "      <td>Coronal plain computed tomography image showi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ROCO_00016</td>\n",
       "      <td>PMC5665693_cureus-0009-00000001639-i01.jpg</td>\n",
       "      <td>Axial source image from an intracranial magne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ROCO_00025</td>\n",
       "      <td>PMC4813433_EJD-10-188-g001.jpg</td>\n",
       "      <td>The apical height, homogeneity, and the thick...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ROCO_00031</td>\n",
       "      <td>PMC4252315_PWKI-10-23517-g001.jpg</td>\n",
       "      <td>CTO of RCA (closure in the 2nd segment)\\n</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id                                       image  \\\n",
       "0  ROCO_00001            PMC4608653_cro-0008-0385-g01.jpg   \n",
       "1  ROCO_00006            PMC4840839_ol-11-05-3298-g02.jpg   \n",
       "2  ROCO_00016  PMC5665693_cureus-0009-00000001639-i01.jpg   \n",
       "3  ROCO_00025              PMC4813433_EJD-10-188-g001.jpg   \n",
       "4  ROCO_00031           PMC4252315_PWKI-10-23517-g001.jpg   \n",
       "\n",
       "                                             caption  \n",
       "0                        Axial MRI (coronal view).\\n  \n",
       "1   Coronal plain computed tomography image showi...  \n",
       "2   Axial source image from an intracranial magne...  \n",
       "3   The apical height, homogeneity, and the thick...  \n",
       "4          CTO of RCA (closure in the 2nd segment)\\n  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add Dir_Path to `name`\n",
    "因为 `open_clip` 的 `csv-img-key` 字段要求直接对应图片的地址."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_path(data, dir_path):\n",
    "    def func(row, dir_path):\n",
    "        row.image = dir_path / \"images\" / row.image\n",
    "        return row\n",
    "    processed_data = data.copy(deep=True)\n",
    "    processed_data = processed_data.apply(lambda x: func(x, dir_path), axis=1)\n",
    "    return processed_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_train, processed_valid, processed_test = add_path(train_data, train_dir_path), add_path(valid_data, valid_dir_path), add_path(test_data, test_dir_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>image</th>\n",
       "      <th>caption</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ROCO_00002</td>\n",
       "      <td>/remote-home/share/medical/public/ROCO/train/r...</td>\n",
       "      <td>Computed tomography scan in axial view showin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROCO_00003</td>\n",
       "      <td>/remote-home/share/medical/public/ROCO/train/r...</td>\n",
       "      <td>Bacterial contamination occurred after comple...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ROCO_00004</td>\n",
       "      <td>/remote-home/share/medical/public/ROCO/train/r...</td>\n",
       "      <td>The patient had residual paralysis of the han...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ROCO_00005</td>\n",
       "      <td>/remote-home/share/medical/public/ROCO/train/r...</td>\n",
       "      <td>Panoramic radiograph after immediate loading.\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ROCO_00007</td>\n",
       "      <td>/remote-home/share/medical/public/ROCO/train/r...</td>\n",
       "      <td>Plain abdomen x-ray: Multiple air levels at t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id                                              image  \\\n",
       "0  ROCO_00002  /remote-home/share/medical/public/ROCO/train/r...   \n",
       "1  ROCO_00003  /remote-home/share/medical/public/ROCO/train/r...   \n",
       "2  ROCO_00004  /remote-home/share/medical/public/ROCO/train/r...   \n",
       "3  ROCO_00005  /remote-home/share/medical/public/ROCO/train/r...   \n",
       "4  ROCO_00007  /remote-home/share/medical/public/ROCO/train/r...   \n",
       "\n",
       "                                             caption  \n",
       "0   Computed tomography scan in axial view showin...  \n",
       "1   Bacterial contamination occurred after comple...  \n",
       "2   The patient had residual paralysis of the han...  \n",
       "3    Panoramic radiograph after immediate loading.\\n  \n",
       "4   Plain abdomen x-ray: Multiple air levels at t...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Verify Data Integrity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter Out Non-exist Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter out non-exist images\n",
    "def filter_nonexist(processed_data):\n",
    "    nonexist_images = []\n",
    "    pbar = tqdm(total=len(processed_data))\n",
    "    for index, row in tqdm(processed_data.iterrows()):\n",
    "        image_path = row['image']\n",
    "        if not image_path.exists():\n",
    "            nonexist_images.append(index)\n",
    "        pbar.update(1)\n",
    "    pbar.close()\n",
    "    return nonexist_images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65450it [03:41, 295.53it/s]0 [03:41<00:00, 457.97it/s]\n",
      "100%|██████████| 65450/65450 [03:41<00:00, 295.53it/s]\n",
      "8180it [00:23, 355.02it/s] [00:22<00:00, 329.99it/s]\n",
      "100%|██████████| 8180/8180 [00:23<00:00, 354.98it/s]\n",
      "8179it [00:24, 338.97it/s] [00:24<00:00, 396.90it/s]\n",
      "100%|██████████| 8179/8179 [00:24<00:00, 338.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non-Exist Train: [6188, 6348, 9369, 13475, 17862, 19678, 19938, 20339, 21050, 21204, 21731, 23560, 27405, 28359, 29971, 30386, 40425, 40560, 42465, 44088, 46072, 48325, 50466, 50719, 53736, 56448, 60110, 61449, 62136, 63143]\n",
      "Non-Exist Valid: [1015, 2451, 2899, 5055, 5819]\n",
      "Non-Exist Test: [4354, 5235, 6471]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "nonexist_train, nonexist_valid, nonexist_test = filter_nonexist(processed_train), filter_nonexist(processed_valid), filter_nonexist(processed_test)\n",
    "print(f\"Non-Exist Train: {nonexist_train}\")\n",
    "print(f\"Non-Exist Valid: {nonexist_valid}\")\n",
    "print(f\"Non-Exist Test: {nonexist_test}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_train.drop(index=nonexist_train, inplace=True)\n",
    "processed_valid.drop(index=nonexist_valid, inplace=True)\n",
    "processed_test.drop(index=nonexist_test, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed Train: 65420\n",
      "Processed Valid: 8175\n",
      "Processed Test: 8176\n"
     ]
    }
   ],
   "source": [
    "print(f\"Processed Train: {len(processed_train)}\")\n",
    "print(f\"Processed Valid: {len(processed_valid)}\")\n",
    "print(f\"Processed Test: {len(processed_test)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter Out Broken Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import PIL\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_broken(processed_data):\n",
    "    broken_images = []\n",
    "    pbar = tqdm(total=len(processed_data))\n",
    "    for index, row in tqdm(processed_data.iterrows()):\n",
    "        image_path = row['image']\n",
    "        try:\n",
    "            image = Image.open(image_path)\n",
    "        except PIL.UnidentifiedImageError:\n",
    "            broken_images.append(index)\n",
    "        pbar.update(1)\n",
    "    pbar.close()\n",
    "    return broken_images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65420it [05:52, 185.56it/s]0 [05:52<00:00, 187.83it/s]\n",
      "100%|██████████| 65420/65420 [05:52<00:00, 185.56it/s]\n",
      "8175it [00:42, 192.80it/s] [00:42<00:00, 200.57it/s]\n",
      "100%|██████████| 8175/8175 [00:42<00:00, 192.79it/s]\n",
      "8176it [00:43, 186.33it/s] [00:43<00:00, 189.89it/s]\n",
      "100%|██████████| 8176/8176 [00:43<00:00, 186.32it/s]\n"
     ]
    }
   ],
   "source": [
    "broken_train = filter_broken(processed_train)\n",
    "broken_valid = filter_broken(processed_valid)\n",
    "broken_test = filter_broken(processed_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "broken_train: [21411]\n",
      "broken_valid: []\n",
      "broken_test: []\n"
     ]
    }
   ],
   "source": [
    "print(f\"broken_train: {broken_train}\")\n",
    "print(f\"broken_valid: {broken_valid}\")\n",
    "print(f\"broken_test: {broken_test}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_train.drop(index=broken_train, inplace=True)\n",
    "processed_valid.drop(index=broken_valid, inplace=True)\n",
    "processed_test.drop(index=broken_test, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_train.to_csv(train_dir_path / \"processed_train.csv\", sep=',', index=False)\n",
    "processed_valid.to_csv(valid_dir_path / \"processed_valid.csv\", sep=',', index=False)\n",
    "processed_test.to_csv(test_dir_path / \"processed_test.csv\", sep=',', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
